
import nltk
from nltk.corpus import brown
from nltk.stem.porter import *
from nltk import pos_tag, word_tokenize

class WnStemmer:
	_stopWords = [
		'in',
		'out',
		'the',
		'from',
		'within',
		'without',
		'for',
		'and',
		'up',
		'down',
		'log',
		'new',
		'more',
		'less',
		'free',
		'sign',
		'to',
		'welcome',
		'too',
		'at',
		'low','high',
		'com',
		'or',
	]
	_removeCharacters = [
		',','.','-','@','/','|',"'",'"',')','(','!','#','$','%','^','&','*','_','+','=','>','<',']','[','}','{','\xc2', '\xae'
	]

	_train_data = None
	_tagger = None

	def __init__(self):
		self._train_data = brown.tagged_sents(categories='news')
		self._regexpTagger = nltk.RegexpTagger(
		[(r'^-?[0-9]+(.[0-9]+)?$', 'CD'),   # cardinal numbers
			(r'(The|the|A|a|An|an)$', 'AT'),   # articles
			(r'.*able$', 'JJ'),                # adjectives
			(r'.*ness$', 'NN'),                # nouns formed from adjectives
			(r'.*ly$', 'RB'),                  # adverbs
			(r'.*s$', 'NNS'),                  # plural nouns
			(r'.*ing$', 'VBG'),                # gerunds
			(r'.*ed$', 'VBD'),                 # past tense verbs
			(r'.*', 'NN')                      # nouns (default)
		])
		self._unigramTagger = nltk.UnigramTagger(self._train_data, backoff=self._regexpTagger)
		self._bigramTagger = nltk.BigramTagger(self._train_data, backoff=self._unigramTagger)
		self._trigramTagger = nltk.TrigramTagger(self._train_data, backoff=self._bigramTagger)
		self._stemmer = PorterStemmer()

	def prepareText(self, text):
		self._text = ' '+text.lower()+' '
		self._text = self._text.replace("  ", " ")
		self.removeCharacters()
		self.removeStopWords()
		return self.tag(text)
	
	def tag(self, text):
		self._tagged = self._trigramTagger.tag(word_tokenize(self._text))
		return self._tagged

	def stem(self, text):
		stemmed = ""

		for i in text.split(" "):
			stemmed+=self._stemmer.stem(i) + " "
		return stemmed
	
	def removeStopWords(self):
		for c in self._stopWords:
			self._text = self._text.replace( ' '+c+' ', ' ' )
		return self._text

	def removeCharacters(self):
		for c in self._removeCharacters:
			self._text = self._text.replace( c, ' ' )
		return self._text

	def getWords(self, partOfSpeach=None):
		words = []
		for i in self._tagged:
			if partOfSpeach==None or i[1]==partOfSpeach:
				words.append(i[0])
		return words